package com.chen.pageProcessor;


import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;

import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.*;

/**
 * @author 陈俊宏
 */
public class DemoProcessor implements PageProcessor {
    private final Site site = Site.me().setRetryTimes(3).setSleepTime(1000);
    private final ItemProcessor itemProcessor;
    static int fail = 0;

    public DemoProcessor(ItemProcessor itemProcessor) {
        this.itemProcessor = itemProcessor;
    }

    @Override
    public void process(Page page) {
        if (page.getHtml().regex("window.location.href").match()) {
            Request request = page.getRequest();
            System.out.println("被要求登录了次数:" + (++fail));
            System.out.println(page.getHtml().toString());
            System.out.println(page.getRequest());
            request.putExtra("retry", "true");
            page.addTargetRequest(request);
            return;
        }
        String type = page.getUrl().regex("[0-9]+,[0-9]+,[0-9]+").get();
        Integer pages = page.getRequest().getExtra("page");
        Integer sNumber = page.getRequest().getExtra("s");
        Integer target = page.getRequest().getExtra("target");
        String brand = page.getRequest().getExtra("brand");
        if (pages > target * 2) {
            return;
        }
        if (pages == 1) {
            //每类的第一个
            int number = 1;
            int targets = Integer.parseInt(page.getHtml().xpath("//div[@id=\"J_topPage\"]/span/i/text()").get());
            List<String> all = page.getHtml().xpath("//li[@class=\"gl-item\"]").all();
            for (String s : all) {
                Html temp = Html.create(s);
                if (!temp.xpath("li[@class=gl-item]/@ware-type").get().equals("0")) {
                    number++;
                }
            }
            Request request = null;
            try {
                request = getReuest(page.getUrl().get()).setUrl("https://list.jd.com/listNew.php?cat=" + type + "&ev=exbrand_" + URLEncoder.encode(brand + "^", "utf-8") + "&scrolling=y&log_id=" + System.currentTimeMillis() + "&isList=1" + "&page=" + 2 + "&s=" + number);
            } catch (UnsupportedEncodingException e) {
                e.printStackTrace();
            }
            request.putExtra("page", 2);
            request.putExtra("target", targets);
            request.putExtra("s", number);
            request.putExtra("type", page.getRequest().getExtra("type").toString());
            request.putExtra("brand", page.getRequest().getExtra("brand").toString());
            request.setPriority(5);
            page.addTargetRequest(request);
        }

        if (page.getUrl().regex("scrolling").match()) {
            Request request = null;
            try {
                request = getReuest(page.getUrl().get()).setUrl("https://list.jd.com/listNew.php?cat=" + type + "&ev=exbrand_" + URLEncoder.encode(brand + "^", "utf-8") + "&page=" + (pages + 1) + "&s=" + (sNumber + 30) + "&click=0");
            } catch (UnsupportedEncodingException e) {
                e.printStackTrace();
            }
            request.putExtra("page", pages + 1);
            request.putExtra("s", sNumber + 30);
            request.putExtra("target", page.getRequest().getExtra("target"));
            request.putExtra("type", page.getRequest().getExtra("type").toString());
            request.putExtra("brand", page.getRequest().getExtra("brand").toString());
            request.setPriority(5);
            page.addTargetRequest(request);
        }

        if (page.getUrl().regex("click").match()) {
            Request request = null;
            try {
                request = getReuest(page.getUrl().get()).setUrl("https://list.jd.com/listNew.php?cat=" + type + "&ev=exbrand_" + URLEncoder.encode(brand + "^", "utf-8") + "&scrolling=y&log_id=" + (System.currentTimeMillis() + 500) + "&isList=1" + "&page=" + (pages + 1) + "&s=" + (sNumber + 30));
            } catch (UnsupportedEncodingException e) {
                e.printStackTrace();
            }
            request.putExtra("page", pages + 1);
            request.putExtra("s", sNumber + 30);
            request.putExtra("target", page.getRequest().getExtra("target"));
            request.putExtra("type", page.getRequest().getExtra("type").toString());
            request.putExtra("brand", page.getRequest().getExtra("brand").toString());
            request.setPriority(5);
            page.addTargetRequest(request);
        }
        itemProcessor.process(page);
    }

    @Override
    public Site getSite() {
        return this.site;
    }

    public Request getReuest(String url) {
        Request request = new Request();
        request.addHeader("Accept-Encoding", "gzip, deflate, br");
        request.addHeader("Connection", "keep-alive");
        request.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:85.0) Gecko/20100101 Firefox/85.0");
        request.addHeader("Host", "list.jd.com");
        request.addHeader("X-Requested-With", "XMLHttpRequest");
        request.addHeader("Pragma", "no-cache");
        request.addHeader("Referer", url);


        return request;
    }

}
